<!DOCTYPE HTML>
<html lang="en" >
    <!-- Start book Python爬虫课程讲义 -->
    <head>
        <!-- head:start -->
        <meta charset="UTF-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <title>多线程爬虫案例 | Python爬虫课程讲义</title>
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <meta name="description" content="">
        <meta name="generator" content="GitBook 2.6.7">
        <meta name="author" content="BigCat">
        
        <meta name="HandheldFriendly" content="true"/>
        <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
        <meta name="apple-mobile-web-app-capable" content="yes">
        <meta name="apple-mobile-web-app-status-bar-style" content="black">
        <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
        <link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
        
    <link rel="stylesheet" href="../../gitbook/style.css">
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-tbfed-pagefooter/footer.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-splitter/splitter.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-highlight/website.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-fontsettings/website.css">
        
    
    

        
    
    
    <link rel="next" href="../../file/part03/3.html" />
    
    
    <link rel="prev" href="../../file/part02/2.8.html" />
    

        <!-- head:end -->
    </head>
    <body>
        <!-- body:start -->
        
    <div class="book"
        data-level="2.9"
        data-chapter-title="多线程爬虫案例"
        data-filepath="file/part02/2.9.md"
        data-basepath="../.."
        data-revision="Thu Feb 09 2017 09:48:59 GMT+0800 (CST)"
        data-innerlanguage="">
    

<div class="book-summary">
    <nav role="navigation">
        <ul class="summary">
            
            
            
            

            

            
    
        <li class="chapter " data-level="0" data-path="index.html">
            
                
                    <a href="../../index.html">
                
                        <i class="fa fa-check"></i>
                        
                        传智播客Python学院爬虫课程
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1" data-path="file/part01/1.html">
            
                
                    <a href="../../file/part01/1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.</b>
                        
                        爬虫原理与数据抓取
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.1" data-path="file/part01/1.1.html">
            
                
                    <a href="../../file/part01/1.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.1.</b>
                        
                        (了解)通用爬虫和聚焦爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="file/part01/1.2.html">
            
                
                    <a href="../../file/part01/1.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.2.</b>
                        
                        (复习)HTTP/HTTPS的请求与响应
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="file/part01/1.3.html">
            
                
                    <a href="../../file/part01/1.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.3.</b>
                        
                        HTTP/HTTPS抓包工具-Fiddler
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="file/part01/1.4.html">
            
                
                    <a href="../../file/part01/1.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.4.</b>
                        
                        urllib2模块的基本使用
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.5" data-path="file/part01/1.5.html">
            
                
                    <a href="../../file/part01/1.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.5.</b>
                        
                        urllib2：GET请求和POST请求
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.6" data-path="file/part01/1.6.html">
            
                
                    <a href="../../file/part01/1.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.6.</b>
                        
                        urllib2：Handler处理器和自定义Opener
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.7" data-path="file/part01/1.7.html">
            
                
                    <a href="../../file/part01/1.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.7.</b>
                        
                        urllib2：URLError与HTTPError
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.8" data-path="file/part01/1.8.html">
            
                
                    <a href="../../file/part01/1.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.8.</b>
                        
                        Requests模块
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2" data-path="file/part02/2.html">
            
                
                    <a href="../../file/part02/2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.</b>
                        
                        非结构化数据与结构化数据提取
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.1" data-path="file/part02/2.1.html">
            
                
                    <a href="../../file/part02/2.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.1.</b>
                        
                        正则表达式re模块
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.2" data-path="file/part02/2.2.html">
            
                
                    <a href="../../file/part02/2.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.2.</b>
                        
                        案例：使用正则表达式的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.3" data-path="file/part02/2.3.html">
            
                
                    <a href="../../file/part02/2.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.3.</b>
                        
                        XPath与lxml类库
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.4" data-path="file/part02/2.4.html">
            
                
                    <a href="../../file/part02/2.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.4.</b>
                        
                        案例：使用XPath的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.5" data-path="file/part02/2.5.html">
            
                
                    <a href="../../file/part02/2.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.5.</b>
                        
                        BeautifulSoup4 解析器
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.6" data-path="file/part02/2.6.html">
            
                
                    <a href="../../file/part02/2.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.6.</b>
                        
                        案例：使用bs4的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.7" data-path="file/part02/2.7.html">
            
                
                    <a href="../../file/part02/2.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.7.</b>
                        
                        JSON模块与JsonPath
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.8" data-path="file/part02/2.8.html">
            
                
                    <a href="../../file/part02/2.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.8.</b>
                        
                        糗事百科案例
                    </a>
            
            
        </li>
    
        <li class="chapter active" data-level="2.9" data-path="file/part02/2.9.html">
            
                
                    <a href="../../file/part02/2.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.9.</b>
                        
                        多线程爬虫案例
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="3" data-path="file/part03/3.html">
            
                
                    <a href="../../file/part03/3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.</b>
                        
                        动态HTML处理和机器图像识别
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="3.1" data-path="file/part03/3.1.html">
            
                
                    <a href="../../file/part03/3.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.1.</b>
                        
                        动态HTML介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.2" data-path="file/part03/3.2.html">
            
                
                    <a href="../../file/part03/3.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.2.</b>
                        
                        Selenium与PhantomJS
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.3" data-path="file/part03/3.3.html">
            
                
                    <a href="../../file/part03/3.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.3.</b>
                        
                        案例一：网站模拟登录
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.4" data-path="file/part03/3.4.html">
            
                
                    <a href="../../file/part03/3.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.4.</b>
                        
                        案例二：动态页面模拟点击
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.5" data-path="file/part03/3.5.html">
            
                
                    <a href="../../file/part03/3.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.5.</b>
                        
                        案例三：执行JavaScript语句
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.6" data-path="file/part03/3.6.html">
            
                
                    <a href="../../file/part03/3.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.6.</b>
                        
                        机器视觉与Tesseract介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.7" data-path="file/part03/3.7.html">
            
                
                    <a href="../../file/part03/3.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.7.</b>
                        
                        处理一些格式规范的文字
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.8" data-path="file/part03/3.8.html">
            
                
                    <a href="../../file/part03/3.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.8.</b>
                        
                        案例：尝试对验证码进行机器识别处理
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.9" data-path="file/part03/3.9.html">
            
                
                    <a href="../../file/part03/3.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.9.</b>
                        
                        机器学习：训练Tesseract
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="4" data-path="file/part04/4.html">
            
                
                    <a href="../../file/part04/4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.</b>
                        
                        Scrapy框架
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="4.1" data-path="file/part04/4.1.html">
            
                
                    <a href="../../file/part04/4.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.1.</b>
                        
                        配置安装
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.2" data-path="file/part04/4.2.html">
            
                
                    <a href="../../file/part04/4.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.2.</b>
                        
                        入门案例
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.3" data-path="file/part04/4.3.html">
            
                
                    <a href="../../file/part04/4.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.3.</b>
                        
                        Scrapy Shell
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.4" data-path="file/part04/4.4.html">
            
                
                    <a href="../../file/part04/4.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.4.</b>
                        
                        Item Pipeline
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.5" data-path="file/part04/4.5.html">
            
                
                    <a href="../../file/part04/4.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.5.</b>
                        
                        Spiders
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.6" data-path="file/part04/4.6.html">
            
                
                    <a href="../../file/part04/4.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.6.</b>
                        
                        CrawlSpiders
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.7" data-path="file/part04/4.7.html">
            
                
                    <a href="../../file/part04/4.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.7.</b>
                        
                        Request/Response
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.8" data-path="file/part04/4.8.html">
            
                
                    <a href="../../file/part04/4.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.8.</b>
                        
                        Downloader Middlewares
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.9" data-path="file/part04/4.9.html">
            
                
                    <a href="../../file/part04/4.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.9.</b>
                        
                        Settings
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="5" data-path="file/part05/5.html">
            
                
                    <a href="../../file/part05/5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.</b>
                        
                        Scrapy实战项目
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="5.1" data-path="file/part05/5.1.html">
            
                
                    <a href="../../file/part05/5.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.1.</b>
                        
                        (案例一)手机App抓包爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.2" data-path="file/part05/5.2.html">
            
                
                    <a href="../../file/part05/5.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.2.</b>
                        
                        (案例二)阳光热线问政平台爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.3" data-path="file/part05/5.3.html">
            
                
                    <a href="../../file/part05/5.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.3.</b>
                        
                        (案例三)新浪网分类资讯爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.4" data-path="file/part05/5.4.html">
            
                
                    <a href="../../file/part05/5.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.4.</b>
                        
                        (案例四)图片下载器爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.5" data-path="file/part05/5.5.html">
            
                
                    <a href="../../file/part05/5.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.5.</b>
                        
                        (案例五)将数据保存在MongoDB中
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.6" data-path="file/part05/5.6.html">
            
                
                    <a href="../../file/part05/5.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.6.</b>
                        
                        (案例六)三种scrapy模拟登陆策略
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.7" data-path="file/part05/5.7.html">
            
                
                    <a href="../../file/part05/5.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.7.</b>
                        
                        附：通过Fiddler进行手机抓包方法
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="6" data-path="file/part06/6.html">
            
                
                    <a href="../../file/part06/6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.</b>
                        
                        scrapy-redis分布式组件
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="6.1" data-path="file/part06/6.1.html">
            
                
                    <a href="../../file/part06/6.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.1.</b>
                        
                        源码分析参考：Connection
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.2" data-path="file/part06/6.2.html">
            
                
                    <a href="../../file/part06/6.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.2.</b>
                        
                        源码分析参考：Dupefilter
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.3" data-path="file/part06/6.3.html">
            
                
                    <a href="../../file/part06/6.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.3.</b>
                        
                        源码分析参考：Picklecompat
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.4" data-path="file/part06/6.4.html">
            
                
                    <a href="../../file/part06/6.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.4.</b>
                        
                        源码分析参考：Pipelines
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.5" data-path="file/part06/6.5.html">
            
                
                    <a href="../../file/part06/6.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.5.</b>
                        
                        源码分析参考：Queue
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.6" data-path="file/part06/6.6.html">
            
                
                    <a href="../../file/part06/6.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.6.</b>
                        
                        源码分析参考：Scheduler
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.7" data-path="file/part06/6.7.html">
            
                
                    <a href="../../file/part06/6.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.7.</b>
                        
                        源码分析参考：Spider
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="7" data-path="file/part07/7.html">
            
                
                    <a href="../../file/part07/7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.</b>
                        
                        scrapy-redis实战
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="7.1" data-path="file/part07/7.1.html">
            
                
                    <a href="../../file/part07/7.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.1.</b>
                        
                        源码自带项目说明
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.2" data-path="file/part07/7.2.html">
            
                
                    <a href="../../file/part07/7.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.2.</b>
                        
                        有缘网分布式爬虫项目1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.3" data-path="file/part07/7.3.html">
            
                
                    <a href="../../file/part07/7.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.3.</b>
                        
                        有缘网分布式爬虫项目2
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.4" data-path="file/part07/7.4.html">
            
                
                    <a href="../../file/part07/7.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.4.</b>
                        
                        处理Redis里的数据
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.5" data-path="file/part07/7.5.html">
            
                
                    <a href="../../file/part07/7.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.5.</b>
                        
                        尝试改写新浪网分类资讯爬虫1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.6" data-path="file/part07/7.6.html">
            
                
                    <a href="../../file/part07/7.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.6.</b>
                        
                        尝试改写新浪网分类资讯爬虫2
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.7" data-path="file/part07/7.7.html">
            
                
                    <a href="../../file/part07/7.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.7.</b>
                        
                        IT桔子分布式项目1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.8" data-path="file/part07/7.8.html">
            
                
                    <a href="../../file/part07/7.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.8.</b>
                        
                        IT桔子分布式项目2
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="8" data-path="file/duanzi/duanzi.html">
            
                
                    <a href="../../file/duanzi/duanzi.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>8.</b>
                        
                        课余段子
                    </a>
            
            
        </li>
    


            
            <li class="divider"></li>
            <li>
                <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
                    Published with GitBook
                </a>
            </li>
            
        </ul>
    </nav>
</div>

    <div class="book-body">
        <div class="body-inner">
            <div class="book-header" role="navigation">
    <!-- Actions Left -->
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href="../../" >Python爬虫课程讲义</a>
    </h1>
</div>

            <div class="page-wrapper" tabindex="-1" role="main">
                <div class="page-inner">
                
                
                    <section class="normal" id="section-">
                    
                        <h1 id="&#x591A;&#x7EBF;&#x7A0B;&#x7CD7;&#x4E8B;&#x767E;&#x79D1;&#x6848;&#x4F8B;">&#x591A;&#x7EBF;&#x7A0B;&#x7CD7;&#x4E8B;&#x767E;&#x79D1;&#x6848;&#x4F8B;</h1>
<p>&#x6848;&#x4F8B;&#x8981;&#x6C42;&#x53C2;&#x8003;&#x4E0A;&#x4E00;&#x4E2A;&#x7CD7;&#x4E8B;&#x767E;&#x79D1;&#x5355;&#x8FDB;&#x7A0B;&#x6848;&#x4F8B;</p>
<h3 id="queue&#xFF08;&#x961F;&#x5217;&#x5BF9;&#x8C61;&#xFF09;">Queue&#xFF08;&#x961F;&#x5217;&#x5BF9;&#x8C61;&#xFF09;</h3>
<p>Queue&#x662F;python&#x4E2D;&#x7684;&#x6807;&#x51C6;&#x5E93;&#xFF0C;&#x53EF;&#x4EE5;&#x76F4;&#x63A5;import Queue&#x5F15;&#x7528;;&#x961F;&#x5217;&#x662F;&#x7EBF;&#x7A0B;&#x95F4;&#x6700;&#x5E38;&#x7528;&#x7684;&#x4EA4;&#x6362;&#x6570;&#x636E;&#x7684;&#x5F62;&#x5F0F;</p>
<p>python&#x4E0B;&#x591A;&#x7EBF;&#x7A0B;&#x7684;&#x601D;&#x8003;</p>
<p>&#x5BF9;&#x4E8E;&#x8D44;&#x6E90;&#xFF0C;&#x52A0;&#x9501;&#x662F;&#x4E2A;&#x91CD;&#x8981;&#x7684;&#x73AF;&#x8282;&#x3002;&#x56E0;&#x4E3A;python&#x539F;&#x751F;&#x7684;list,dict&#x7B49;&#xFF0C;&#x90FD;&#x662F;not thread safe&#x7684;&#x3002;&#x800C;Queue&#xFF0C;&#x662F;&#x7EBF;&#x7A0B;&#x5B89;&#x5168;&#x7684;&#xFF0C;&#x56E0;&#x6B64;&#x5728;&#x6EE1;&#x8DB3;&#x4F7F;&#x7528;&#x6761;&#x4EF6;&#x4E0B;&#xFF0C;&#x5EFA;&#x8BAE;&#x4F7F;&#x7528;&#x961F;&#x5217;</p>
<ol>
<li><p>&#x521D;&#x59CB;&#x5316;&#xFF1A; class Queue.Queue(maxsize) FIFO &#x5148;&#x8FDB;&#x5148;&#x51FA;</p>
</li>
<li><p>&#x5305;&#x4E2D;&#x7684;&#x5E38;&#x7528;&#x65B9;&#x6CD5;:</p>
<ul>
<li><p>Queue.qsize() &#x8FD4;&#x56DE;&#x961F;&#x5217;&#x7684;&#x5927;&#x5C0F;</p>
</li>
<li><p>Queue.empty() &#x5982;&#x679C;&#x961F;&#x5217;&#x4E3A;&#x7A7A;&#xFF0C;&#x8FD4;&#x56DE;True,&#x53CD;&#x4E4B;False</p>
</li>
<li><p>Queue.full() &#x5982;&#x679C;&#x961F;&#x5217;&#x6EE1;&#x4E86;&#xFF0C;&#x8FD4;&#x56DE;True,&#x53CD;&#x4E4B;False</p>
</li>
<li><p>Queue.full &#x4E0E; maxsize &#x5927;&#x5C0F;&#x5BF9;&#x5E94;</p>
</li>
<li><p>Queue.get([block[, timeout]])&#x83B7;&#x53D6;&#x961F;&#x5217;&#xFF0C;timeout&#x7B49;&#x5F85;&#x65F6;&#x95F4;</p>
</li>
</ul>
</li>
<li><p>&#x521B;&#x5EFA;&#x4E00;&#x4E2A;&#x201C;&#x961F;&#x5217;&#x201D;&#x5BF9;&#x8C61;</p>
<ul>
<li>import Queue </li>
<li>myqueue = Queue.Queue(maxsize = 10)</li>
</ul>
</li>
<li><p>&#x5C06;&#x4E00;&#x4E2A;&#x503C;&#x653E;&#x5165;&#x961F;&#x5217;&#x4E2D;</p>
<ul>
<li>myqueue.put(10)</li>
</ul>
</li>
<li><p>&#x5C06;&#x4E00;&#x4E2A;&#x503C;&#x4ECE;&#x961F;&#x5217;&#x4E2D;&#x53D6;&#x51FA;</p>
<ul>
<li>myqueue.get()</li>
</ul>
</li>
</ol>
<h3 id="&#x591A;&#x7EBF;&#x7A0B;&#x793A;&#x610F;&#x56FE;">&#x591A;&#x7EBF;&#x7A0B;&#x793A;&#x610F;&#x56FE;</h3>
<p><img src="../images/03-mulithread.png" alt=""></p>
<pre><code class="lang-python"><span class="hljs-comment"># -*- coding:utf-8 -*-</span>
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> lxml <span class="hljs-keyword">import</span> etree
<span class="hljs-keyword">from</span> Queue <span class="hljs-keyword">import</span> Queue
<span class="hljs-keyword">import</span> threading
<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> json


<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">thread_crawl</span><span class="hljs-params">(threading.Thread)</span>:</span>
    <span class="hljs-string">&apos;&apos;&apos;
    &#x6293;&#x53D6;&#x7EBF;&#x7A0B;&#x7C7B;
    &apos;&apos;&apos;</span>

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self, threadID, q)</span>:</span>
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.q = q

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">run</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-keyword">print</span> <span class="hljs-string">&quot;Starting &quot;</span> + self.threadID
        self.qiushi_spider()
        <span class="hljs-keyword">print</span> <span class="hljs-string">&quot;Exiting &quot;</span>, self.threadID

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">qiushi_spider</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-comment"># page = 1</span>
        <span class="hljs-keyword">while</span> <span class="hljs-keyword">True</span>:
            <span class="hljs-keyword">if</span> self.q.empty():
                <span class="hljs-keyword">break</span>
            <span class="hljs-keyword">else</span>:
                page = self.q.get()
                <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;qiushi_spider=&apos;</span>, self.threadID, <span class="hljs-string">&apos;,page=&apos;</span>, str(page)
                url = <span class="hljs-string">&apos;http://www.qiushibaike.com/8hr/page/&apos;</span> + str(page) + <span class="hljs-string">&apos;/&apos;</span>
                headers = {
                    <span class="hljs-string">&apos;User-Agent&apos;</span>: <span class="hljs-string">&apos;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36&apos;</span>,
                    <span class="hljs-string">&apos;Accept-Language&apos;</span>: <span class="hljs-string">&apos;zh-CN,zh;q=0.8&apos;</span>}
                <span class="hljs-comment"># &#x591A;&#x6B21;&#x5C1D;&#x8BD5;&#x5931;&#x8D25;&#x7ED3;&#x675F;&#x3001;&#x9632;&#x6B62;&#x6B7B;&#x5FAA;&#x73AF;</span>
                timeout = <span class="hljs-number">4</span>
                <span class="hljs-keyword">while</span> timeout &gt; <span class="hljs-number">0</span>:
                    timeout -= <span class="hljs-number">1</span>
                    <span class="hljs-keyword">try</span>:
                        content = requests.get(url, headers=headers)
                        data_queue.put(content.text)
                        <span class="hljs-keyword">break</span>
                    <span class="hljs-keyword">except</span> Exception, e:
                        <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;qiushi_spider&apos;</span>, e
                <span class="hljs-keyword">if</span> timeout &lt; <span class="hljs-number">0</span>:
                    <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;timeout&apos;</span>, url


<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Thread_Parser</span><span class="hljs-params">(threading.Thread)</span>:</span>
    <span class="hljs-string">&apos;&apos;&apos;
    &#x9875;&#x9762;&#x89E3;&#x6790;&#x7C7B;&#xFF1B;
    &apos;&apos;&apos;</span>

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self, threadID, queue, lock, f)</span>:</span>
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.queue = queue
        self.lock = lock
        self.f = f

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">run</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;starting &apos;</span>, self.threadID
        <span class="hljs-keyword">global</span> total, exitFlag_Parser
        <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> exitFlag_Parser:
            <span class="hljs-keyword">try</span>:
                <span class="hljs-string">&apos;&apos;&apos;
                &#x8C03;&#x7528;&#x961F;&#x5217;&#x5BF9;&#x8C61;&#x7684;get()&#x65B9;&#x6CD5;&#x4ECE;&#x961F;&#x5934;&#x5220;&#x9664;&#x5E76;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x9879;&#x76EE;&#x3002;&#x53EF;&#x9009;&#x53C2;&#x6570;&#x4E3A;block&#xFF0C;&#x9ED8;&#x8BA4;&#x4E3A;True&#x3002;
                &#x5982;&#x679C;&#x961F;&#x5217;&#x4E3A;&#x7A7A;&#x4E14;block&#x4E3A;True&#xFF0C;get()&#x5C31;&#x4F7F;&#x8C03;&#x7528;&#x7EBF;&#x7A0B;&#x6682;&#x505C;&#xFF0C;&#x76F4;&#x81F3;&#x6709;&#x9879;&#x76EE;&#x53EF;&#x7528;&#x3002;
                &#x5982;&#x679C;&#x961F;&#x5217;&#x4E3A;&#x7A7A;&#x4E14;block&#x4E3A;False&#xFF0C;&#x961F;&#x5217;&#x5C06;&#x5F15;&#x53D1;Empty&#x5F02;&#x5E38;&#x3002;
                &apos;&apos;&apos;</span>
                item = self.queue.get(<span class="hljs-keyword">False</span>)
                <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> item:
                    <span class="hljs-keyword">pass</span>
                self.parse_data(item)
                self.queue.task_done()
                <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;Thread_Parser=&apos;</span>, self.threadID, <span class="hljs-string">&apos;,total=&apos;</span>, total
            <span class="hljs-keyword">except</span>:
                <span class="hljs-keyword">pass</span>
        <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;Exiting &apos;</span>, self.threadID

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse_data</span><span class="hljs-params">(self, item)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;
        &#x89E3;&#x6790;&#x7F51;&#x9875;&#x51FD;&#x6570;
        :param item: &#x7F51;&#x9875;&#x5185;&#x5BB9;
        :return:
        &apos;&apos;&apos;</span>
        <span class="hljs-keyword">global</span> total
        <span class="hljs-keyword">try</span>:
            html = etree.HTML(item)
            result = html.xpath(<span class="hljs-string">&apos;//div[contains(@id,&quot;qiushi_tag&quot;)]&apos;</span>)
            <span class="hljs-keyword">for</span> site <span class="hljs-keyword">in</span> result:
                <span class="hljs-keyword">try</span>:
                    imgUrl = site.xpath(<span class="hljs-string">&apos;.//img/@src&apos;</span>)[<span class="hljs-number">0</span>]
                    title = site.xpath(<span class="hljs-string">&apos;.//h2&apos;</span>)[<span class="hljs-number">0</span>].text
                    content = site.xpath(<span class="hljs-string">&apos;.//div[@class=&quot;content&quot;]/span&apos;</span>)[<span class="hljs-number">0</span>].text.strip()
                    vote = <span class="hljs-keyword">None</span>
                    comments = <span class="hljs-keyword">None</span>
                    <span class="hljs-keyword">try</span>:
                        vote = site.xpath(<span class="hljs-string">&apos;.//i&apos;</span>)[<span class="hljs-number">0</span>].text
                        comments = site.xpath(<span class="hljs-string">&apos;.//i&apos;</span>)[<span class="hljs-number">1</span>].text
                    <span class="hljs-keyword">except</span>:
                        <span class="hljs-keyword">pass</span>
                    result = {
                        <span class="hljs-string">&apos;imgUrl&apos;</span>: imgUrl,
                        <span class="hljs-string">&apos;title&apos;</span>: title,
                        <span class="hljs-string">&apos;content&apos;</span>: content,
                        <span class="hljs-string">&apos;vote&apos;</span>: vote,
                        <span class="hljs-string">&apos;comments&apos;</span>: comments,
                    }

                    <span class="hljs-keyword">with</span> self.lock:
                        <span class="hljs-comment"># print &apos;write %s&apos; % json.dumps(result)</span>
                        self.f.write(json.dumps(result, ensure_ascii=<span class="hljs-keyword">False</span>).encode(<span class="hljs-string">&apos;utf-8&apos;</span>) + <span class="hljs-string">&quot;\n&quot;</span>)

                <span class="hljs-keyword">except</span> Exception, e:
                    <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;site in result&apos;</span>, e
        <span class="hljs-keyword">except</span> Exception, e:
            <span class="hljs-keyword">print</span> <span class="hljs-string">&apos;parse_data&apos;</span>, e
        <span class="hljs-keyword">with</span> self.lock:
            total += <span class="hljs-number">1</span>

data_queue = Queue()
exitFlag_Parser = <span class="hljs-keyword">False</span>
lock = threading.Lock()
total = <span class="hljs-number">0</span>

<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">main</span><span class="hljs-params">()</span>:</span>
    output = open(<span class="hljs-string">&apos;qiushibaike.json&apos;</span>, <span class="hljs-string">&apos;a&apos;</span>)

    <span class="hljs-comment">#&#x521D;&#x59CB;&#x5316;&#x7F51;&#x9875;&#x9875;&#x7801;page&#x4ECE;1-10&#x4E2A;&#x9875;&#x9762;</span>
    pageQueue = Queue(<span class="hljs-number">50</span>)
    <span class="hljs-keyword">for</span> page <span class="hljs-keyword">in</span> range(<span class="hljs-number">1</span>, <span class="hljs-number">11</span>):
        pageQueue.put(page)

    <span class="hljs-comment">#&#x521D;&#x59CB;&#x5316;&#x91C7;&#x96C6;&#x7EBF;&#x7A0B;</span>
    crawlthreads = []
    crawlList = [<span class="hljs-string">&quot;crawl-1&quot;</span>, <span class="hljs-string">&quot;crawl-2&quot;</span>, <span class="hljs-string">&quot;crawl-3&quot;</span>]

    <span class="hljs-keyword">for</span> threadID <span class="hljs-keyword">in</span> crawlList:
        thread = thread_crawl(threadID, pageQueue)
        thread.start()
        crawlthreads.append(thread)

    <span class="hljs-comment">#&#x521D;&#x59CB;&#x5316;&#x89E3;&#x6790;&#x7EBF;&#x7A0B;parserList</span>
    parserthreads = []
    parserList = [<span class="hljs-string">&quot;parser-1&quot;</span>, <span class="hljs-string">&quot;parser-2&quot;</span>, <span class="hljs-string">&quot;parser-3&quot;</span>]
    <span class="hljs-comment">#&#x5206;&#x522B;&#x542F;&#x52A8;parserList</span>
    <span class="hljs-keyword">for</span> threadID <span class="hljs-keyword">in</span> parserList:
        thread = Thread_Parser(threadID, data_queue, lock, output)
        thread.start()
        parserthreads.append(thread)

    <span class="hljs-comment"># &#x7B49;&#x5F85;&#x961F;&#x5217;&#x6E05;&#x7A7A;</span>
    <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> pageQueue.empty():
        <span class="hljs-keyword">pass</span>

    <span class="hljs-comment"># &#x7B49;&#x5F85;&#x6240;&#x6709;&#x7EBF;&#x7A0B;&#x5B8C;&#x6210;</span>
    <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> crawlthreads:
        t.join()

    <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> data_queue.empty():
        <span class="hljs-keyword">pass</span>
    <span class="hljs-comment"># &#x901A;&#x77E5;&#x7EBF;&#x7A0B;&#x662F;&#x65F6;&#x5019;&#x9000;&#x51FA;</span>
    <span class="hljs-keyword">global</span> exitFlag_Parser
    exitFlag_Parser = <span class="hljs-keyword">True</span>

    <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> parserthreads:
        t.join()
    <span class="hljs-keyword">print</span> <span class="hljs-string">&quot;Exiting Main Thread&quot;</span>
    <span class="hljs-keyword">with</span> lock:
        output.close()


<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&apos;__main__&apos;</span>:
    main()
</code></pre>
<p><img src="../images/threadqiushibaike.png" alt=""> </p>
<footer class="page-footer"><span class="copyright">Copyright &#xA9; BigCat all right reserved&#xFF0C;powered by Gitbook</span><span class="footer-modification">&#x300C;Revision Time:
2017-01-18 03:21:55&#x300D;
</span></footer>
                    
                    </section>
                
                
                </div>
            </div>
        </div>

        
        <a href="../../file/part02/2.8.html" class="navigation navigation-prev " aria-label="Previous page: 糗事百科案例"><i class="fa fa-angle-left"></i></a>
        
        
        <a href="../../file/part03/3.html" class="navigation navigation-next " aria-label="Next page: 动态HTML处理和机器图像识别"><i class="fa fa-angle-right"></i></a>
        
    </div>
</div>

        
<script src="../../gitbook/app.js"></script>

    
    <script src="../../gitbook/plugins/gitbook-plugin-splitter/splitter.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-fontsettings/buttons.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-livereload/plugin.js"></script>
    

<script>
require(["gitbook"], function(gitbook) {
    var config = {"disqus":{"shortName":"gitbookuse"},"github":{"url":"https://github.com/dododream"},"search-pro":{"cutWordLib":"nodejieba","defineWord":["gitbook-use"]},"sharing":{"weibo":true,"facebook":true,"twitter":true,"google":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"tbfed-pagefooter":{"copyright":"Copyright © BigCat","modify_label":"「Revision Time:","modify_format":"YYYY-MM-DD HH:mm:ss」"},"baidu":{"token":"ff100361cdce95dd4c8fb96b4009f7bc"},"sitemap":{"hostname":"http://www.treenewbee.top"},"donate":{"wechat":"http://weixin.png","alipay":"http://alipay.png","title":"","button":"赏","alipayText":"支付宝打赏","wechatText":"微信打赏"},"edit-link":{"base":"https://github.com/dododream/edit","label":"Edit This Page"},"splitter":{},"toggle-chapters":{},"highlight":{},"fontsettings":{"theme":"white","family":"sans","size":2},"livereload":{}};
    gitbook.start(config);
});
</script>

        <!-- body:end -->
    </body>
    <!-- End of book Python爬虫课程讲义 -->
</html>
